{
 "metadata": {
  "name": "",
  "signature": "sha256:ad92d22461092ed2991a0374c5d5187892479b0db395a0e1a6d7692b3fc3b6aa"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import graphlab as gl\n",
      "ratings = gl.SFrame.read_csv(\"../../data/netflix/netflix_mm.train\", delimiter=\"\\t\", header=False,\n",
      "                             column_type_hints=[int,int,int]);\n",
      "ratings.rename({'X1':'movieid', 'X2':'userid', 'X3':'rating'});"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "[INFO] Start server at: ipc:///tmp/graphlab_server-27360 - Server binary: /usr/local/lib/python2.7/dist-packages/graphlab/unity_server - Server log: /tmp/graphlab_server_1431408759.log\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "[INFO] GraphLab Server Version: 1.3.0\n"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: Read 4082259 lines. Lines per second: 4.81022e+06</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: Read 4082259 lines. Lines per second: 4.81022e+06"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: Read 56182627 lines. Lines per second: 9.46167e+06</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: Read 56182627 lines. Lines per second: 9.46167e+06"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: Finished parsing file /code/BIDMach/data/netflix/netflix_mm.train</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: Finished parsing file /code/BIDMach/data/netflix/netflix_mm.train"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: Parsing completed. Parsed 99072124 lines in 10.1339 secs.</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: Parsing completed. Parsed 99072124 lines in 10.1339 secs."
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ratings.head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\"><table frame=\"box\" rules=\"cols\">\n",
        "    <tr>\n",
        "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">movieid</th>\n",
        "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">userid</th>\n",
        "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">rating</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">1</td>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">1</td>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">8</td>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">1</td>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">4</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">17</td>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">1</td>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">30</td>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">1</td>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">44</td>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">1</td>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">58</td>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">1</td>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">76</td>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">1</td>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">80</td>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">1</td>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">81</td>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">1</td>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">83</td>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">1</td>\n",
        "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">3</td>\n",
        "    </tr>\n",
        "</table>\n",
        "[10 rows x 3 columns]<br/>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 2,
       "text": [
        "Columns:\n",
        "\tmovieid\tint\n",
        "\tuserid\tint\n",
        "\trating\tint\n",
        "\n",
        "Rows: 10\n",
        "\n",
        "Data:\n",
        "+---------+--------+--------+\n",
        "| movieid | userid | rating |\n",
        "+---------+--------+--------+\n",
        "|    1    |   1    |   3    |\n",
        "|    8    |   1    |   4    |\n",
        "|    17   |   1    |   2    |\n",
        "|    30   |   1    |   3    |\n",
        "|    44   |   1    |   3    |\n",
        "|    58   |   1    |   5    |\n",
        "|    76   |   1    |   3    |\n",
        "|    80   |   1    |   3    |\n",
        "|    81   |   1    |   3    |\n",
        "|    83   |   1    |   3    |\n",
        "+---------+--------+--------+\n",
        "[10 rows x 3 columns]\n"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ratings['userid'].max()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 3,
       "text": [
        "480189"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "training, testing = ratings.random_split(0.9)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#training, testing = gl.recommender.random_split_by_user(ratings, 'userid', 'movieid', max_num_users=1000, item_test_proportion=0.3)\n",
      "#print training.num_rows(), testing.num_rows()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "m = gl.recommender.factorization_recommender.create(training, 'userid', 'movieid', 'rating',\n",
      "                                                    num_factors=512, max_iterations=15, solver='sgd', \n",
      "                                                    regularization = 1.0e-6, sgd_step_size=0.0)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<pre>PROGRESS: Recsys training: model = factorization_recommender</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: Recsys training: model = factorization_recommender"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: Preparing data set.</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: Preparing data set."
       ]
      },
      {
       "html": [
        "<pre>PROGRESS:     Data has 89161285 observations with 479965 users and 17770 items.</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS:     Data has 89161285 observations with 479965 users and 17770 items."
       ]
      },
      {
       "html": [
        "<pre>PROGRESS:     Data prepared in: 35.6968s</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS:     Data prepared in: 35.6968s"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: Training factorization_recommender for recommendations.</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: Training factorization_recommender for recommendations."
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: +--------------------------------+--------------------------------------------------+----------+</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: +--------------------------------+--------------------------------------------------+----------+"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | Parameter                      | Description                                      | Value    |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | Parameter                      | Description                                      | Value    |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: +--------------------------------+--------------------------------------------------+----------+</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: +--------------------------------+--------------------------------------------------+----------+"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | num_factors                    | Factor Dimension                                 | 512      |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | num_factors                    | Factor Dimension                                 | 512      |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | regularization                 | L2 Regularization on Factors                     | 1e-06    |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | regularization                 | L2 Regularization on Factors                     | 1e-06    |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | solver                         | Solver used for training                         | sgd      |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | solver                         | Solver used for training                         | sgd      |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | linear_regularization          | L2 Regularization on Linear Coefficients         | 1e-10    |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | linear_regularization          | L2 Regularization on Linear Coefficients         | 1e-10    |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | max_iterations                 | Maximum Number of Iterations                     | 15       |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | max_iterations                 | Maximum Number of Iterations                     | 15       |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: +--------------------------------+--------------------------------------------------+----------+</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: +--------------------------------+--------------------------------------------------+----------+"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS:   Optimizing model using SGD; tuning step size.</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS:   Optimizing model using SGD; tuning step size."
       ]
      },
      {
       "html": [
        "<pre>PROGRESS:   Using 11145160 / 89161285 points for tuning the step size.</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS:   Using 11145160 / 89161285 points for tuning the step size."
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: +---------+-------------------+------------------------------------------+</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: +---------+-------------------+------------------------------------------+"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | Attempt | Initial Step Size | Estimated Objective Value                |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | Attempt | Initial Step Size | Estimated Objective Value                |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: +---------+-------------------+------------------------------------------+</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: +---------+-------------------+------------------------------------------+"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 0       | 0.826398          | Not Viable                               |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 0       | 0.826398          | Not Viable                               |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 1       | 0.206599          | Not Viable                               |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 1       | 0.206599          | Not Viable                               |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 2       | 0.0516499         | Not Viable                               |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 2       | 0.0516499         | Not Viable                               |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 3       | 0.0129125         | 0.828073                                 |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 3       | 0.0129125         | 0.828073                                 |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 4       | 0.00645623        | 0.848489                                 |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 4       | 0.00645623        | 0.848489                                 |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 5       | 0.00322812        | 0.875842                                 |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 5       | 0.00322812        | 0.875842                                 |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 6       | 0.00161406        | 0.911026                                 |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 6       | 0.00161406        | 0.911026                                 |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: +---------+-------------------+------------------------------------------+</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: +---------+-------------------+------------------------------------------+"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | Final   | 0.0129125         | 0.828073                                 |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | Final   | 0.0129125         | 0.828073                                 |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: +---------+-------------------+------------------------------------------+</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: +---------+-------------------+------------------------------------------+"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: Starting Optimization.</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: Starting Optimization."
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: +---------+--------------+-------------------+-----------------------+-------------+</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: +---------+--------------+-------------------+-----------------------+-------------+"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | Iter.   | Elapsed Time | Approx. Objective | Approx. Training RMSE | Step Size   |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | Iter.   | Elapsed Time | Approx. Objective | Approx. Training RMSE | Step Size   |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: +---------+--------------+-------------------+-----------------------+-------------+</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: +---------+--------------+-------------------+-----------------------+-------------+"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | Initial | 107us        | 1.1764            | 1.08462               |             |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | Initial | 107us        | 1.1764            | 1.08462               |             |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: +---------+--------------+-------------------+-----------------------+-------------+</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: +---------+--------------+-------------------+-----------------------+-------------+"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 1       | 42.04s       | DIVERGED          | DIVERGED              | 0.0129125   |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 1       | 42.04s       | DIVERGED          | DIVERGED              | 0.0129125   |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | RESET   | 51.97s       | 1.17639           | 1.08462               |             |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | RESET   | 51.97s       | 1.17639           | 1.08462               |             |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 1       | 1m 26s       | 0.868174          | 0.900875              | 0.00645623  |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 1       | 1m 26s       | 0.868174          | 0.900875              | 0.00645623  |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 2       | 2m 1s        | 0.791215          | 0.845199              | 0.0038389   |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 2       | 2m 1s        | 0.791215          | 0.845199              | 0.0038389   |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 3       | 2m 35s       | 0.759365          | 0.8179                | 0.00283229  |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 3       | 2m 35s       | 0.759365          | 0.8179                | 0.00283229  |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 4       | 3m 10s       | 0.744856          | 0.804225              | 0.00228262  |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 4       | 3m 10s       | 0.744856          | 0.804225              | 0.00228262  |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 5       | 3m 45s       | 0.735868          | 0.795077              | 0.00193086  |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 5       | 3m 45s       | 0.735868          | 0.795077              | 0.00193086  |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 6       | 4m 19s       | 0.728657          | 0.787512              | 0.00166474  |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 6       | 4m 19s       | 0.728657          | 0.787512              | 0.00166474  |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 7       | 5m 22s       | 0.722316          | 0.780765              | 0.00144958  |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 7       | 5m 22s       | 0.722316          | 0.780765              | 0.00144958  |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 8       | 6m 27s       | 0.716643          | 0.774656              | 0.00128367  |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 8       | 6m 27s       | 0.716643          | 0.774656              | 0.00128367  |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 9       | 7m 30s       | 0.711653          | 0.769031              | 0.00115184  |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 9       | 7m 30s       | 0.711653          | 0.769031              | 0.00115184  |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 10      | 8m 5s        | 0.707019          | 0.763785              | 0.00104456  |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 10      | 8m 5s        | 0.707019          | 0.763785              | 0.00104456  |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 11      | 8m 39s       | 0.702889          | 0.758958              | 0.000955564 |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 11      | 8m 39s       | 0.702889          | 0.758958              | 0.000955564 |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 12      | 9m 13s       | 0.699133          | 0.754473              | 0.000880543 |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 12      | 9m 13s       | 0.699133          | 0.754473              | 0.000880543 |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 13      | 9m 47s       | 0.6957            | 0.750289              | 0.000816443 |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 13      | 9m 47s       | 0.6957            | 0.750289              | 0.000816443 |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 14      | 10m 21s      | 0.692552          | 0.746361              | 0.000761043 |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 14      | 10m 21s      | 0.692552          | 0.746361              | 0.000761043 |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: | 15      | 10m 55s      | 0.689431          | 0.742615              | 0.000712684 |</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: | 15      | 10m 55s      | 0.689431          | 0.742615              | 0.000712684 |"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: +---------+--------------+-------------------+-----------------------+-------------+</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: +---------+--------------+-------------------+-----------------------+-------------+"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: Optimization Complete: Maximum number of passes through the data reached.</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: Optimization Complete: Maximum number of passes through the data reached."
       ]
      },
      {
       "html": [
        "<pre>PROGRESS: Computing final objective value and training RMSE.</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS: Computing final objective value and training RMSE."
       ]
      },
      {
       "html": [
        "<pre>PROGRESS:        Final objective value: 0.683794</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS:        Final objective value: 0.683794"
       ]
      },
      {
       "html": [
        "<pre>PROGRESS:        Final training RMSE: 0.73881</pre>"
       ],
       "metadata": {},
       "output_type": "display_data",
       "text": [
        "PROGRESS:        Final training RMSE: 0.73881"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Look at model statistics\n",
      "m"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 7,
       "text": [
        "Class                           : FactorizationRecommender\n",
        "\n",
        "Schema\n",
        "------\n",
        "User ID                         : userid\n",
        "Item ID                         : movieid\n",
        "Target                          : rating\n",
        "Additional observation features : 0\n",
        "Number of user side features    : 0\n",
        "Number of item side features    : 0\n",
        "\n",
        "Statistics\n",
        "----------\n",
        "Number of observations          : 89161285\n",
        "Number of users                 : 479965\n",
        "Number of items                 : 17770\n",
        "\n",
        "Training summary\n",
        "----------------\n",
        "Training time                   : 849.7042\n",
        "\n",
        "Settings\n",
        "--------\n",
        "additional_iterations_if_unhealthy: 5\n",
        "sgd_step_adjustment_interval    : 4\n",
        "num_factors                     : 512\n",
        "init_random_sigma               : 0.01\n",
        "max_iterations                  : 15\n",
        "regularization_type             : normal\n",
        "side_data_factorization         : 1\n",
        "regularization                  : 0.0\n",
        "num_tempering_iterations        : 4\n",
        "sgd_step_size                   : 0.0\n",
        "sgd_trial_sample_proportion     : 0.125\n",
        "sgd_sampling_block_size         : 131072\n",
        "binary_target                   : 0\n",
        "nmf                             : 0\n",
        "track_exact_loss                : 0\n",
        "sgd_trial_sample_minimum_size   : 10000\n",
        "sgd_convergence_interval        : 4\n",
        "solver                          : sgd\n",
        "tempering_regularization_start_value: 0.0\n",
        "sgd_convergence_threshold       : 0.0\n",
        "sgd_max_trial_iterations        : 5\n",
        "step_size_decrease_rate         : 0.75\n",
        "linear_regularization           : 0.0\n"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Evaluate rmse (average prediction error) on the test set\n",
      "%time m.evaluate(testing,verbose=False)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "CPU times: user 636 ms, sys: 167 ms, total: 803 ms\n",
        "Wall time: 3min 50s\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 8,
       "text": [
        "{'precision_recall_by_user': Columns:\n",
        "\tuserid\tint\n",
        "\tcutoff\tint\n",
        "\tprecision\tfloat\n",
        "\trecall\tfloat\n",
        "\tcount\tint\n",
        "\n",
        "Rows: 1359441\n",
        "\n",
        "Data:\n",
        "+--------+--------+-----------+-----------------+-------+\n",
        "| userid | cutoff | precision |      recall     | count |\n",
        "+--------+--------+-----------+-----------------+-------+\n",
        "|   1    |   5    |    1.0    | 0.0223214285714 |  224  |\n",
        "|   1    |   10   |    0.7    |     0.03125     |  224  |\n",
        "|   1    |   15   |    0.8    | 0.0535714285714 |  224  |\n",
        "|   2    |   5    |    0.0    |       0.0       |   9   |\n",
        "|   2    |   10   |    0.0    |       0.0       |   9   |\n",
        "|   2    |   15   |    0.0    |       0.0       |   9   |\n",
        "|   3    |   5    |    0.0    |       0.0       |   36  |\n",
        "|   3    |   10   |    0.0    |       0.0       |   36  |\n",
        "|   3    |   15   |    0.0    |       0.0       |   36  |\n",
        "|   4    |   5    |    0.0    |       0.0       |   9   |\n",
        "|  ...   |  ...   |    ...    |       ...       |  ...  |\n",
        "+--------+--------+-----------+-----------------+-------+\n",
        "[1359441 rows x 5 columns]\n",
        "Note: Only the head of the SFrame is printed.\n",
        "You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.,\n",
        " 'precision_recall_overall': Columns:\n",
        "\tcutoff\tint\n",
        "\tprecision\tfloat\n",
        "\trecall\tfloat\n",
        "\n",
        "Rows: 3\n",
        "\n",
        "Data:\n",
        "+--------+-----------------+-----------------+\n",
        "| cutoff |    precision    |      recall     |\n",
        "+--------+-----------------+-----------------+\n",
        "|   5    | 0.0789218509667 |  0.018458748315 |\n",
        "|   10   | 0.0601906224691 | 0.0279574359437 |\n",
        "|   15   |  0.051090705665 |  0.035643976875 |\n",
        "+--------+-----------------+-----------------+\n",
        "[3 rows x 3 columns]\n",
        ",\n",
        " 'rmse_by_item': Columns:\n",
        "\tmovieid\tint\n",
        "\tcount\tint\n",
        "\trmse\tfloat\n",
        "\n",
        "Rows: 17762\n",
        "\n",
        "Data:\n",
        "+---------+-------+----------------+\n",
        "| movieid | count |      rmse      |\n",
        "+---------+-------+----------------+\n",
        "|   7899  |  1411 | 0.776659090298 |\n",
        "|   5288  |   13  | 1.06684819974  |\n",
        "|   5684  |   19  | 1.23791877002  |\n",
        "|  14515  |   97  | 1.04050973948  |\n",
        "|  10273  |  132  | 0.705268251495 |\n",
        "|  11106  |   23  | 0.696915869417 |\n",
        "|   5531  |  228  | 0.825257800539 |\n",
        "|   5921  |   18  | 1.28093222584  |\n",
        "|  14406  |   27  | 0.946851217404 |\n",
        "|   2871  |   42  | 0.954883211501 |\n",
        "|   ...   |  ...  |      ...       |\n",
        "+---------+-------+----------------+\n",
        "[17762 rows x 3 columns]\n",
        "Note: Only the head of the SFrame is printed.\n",
        "You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.,\n",
        " 'rmse_by_user': Columns:\n",
        "\tuserid\tint\n",
        "\tcount\tint\n",
        "\trmse\tfloat\n",
        "\n",
        "Rows: 453147\n",
        "\n",
        "Data:\n",
        "+--------+-------+----------------+\n",
        "| userid | count |      rmse      |\n",
        "+--------+-------+----------------+\n",
        "| 211023 |   5   | 0.440468462295 |\n",
        "| 442699 |   13  | 0.319222179826 |\n",
        "| 79732  |   6   | 0.672370758241 |\n",
        "| 333842 |   63  | 0.735925846462 |\n",
        "|  7899  |   34  | 0.746596218723 |\n",
        "| 25263  |   1   | 0.835913385967 |\n",
        "| 453182 |   15  | 0.459506083838 |\n",
        "| 87629  |   29  | 0.490420055215 |\n",
        "| 459067 |   38  | 0.58144293467  |\n",
        "| 43116  |   31  | 0.778477874692 |\n",
        "|  ...   |  ...  |      ...       |\n",
        "+--------+-------+----------------+\n",
        "[453147 rows x 3 columns]\n",
        "Note: Only the head of the SFrame is printed.\n",
        "You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.,\n",
        " 'rmse_overall': 0.8234641905616967}"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    }
   ],
   "metadata": {}
  }
 ]
}